{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第7章 计算性能\n",
    "### 7.1 命令式编程和符号式编程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def add(a, b):\n",
    "    return a+b\n",
    "def fancy_func(a, b, c, d):\n",
    "    e = add(a, b)\n",
    "    f = add(c, d)\n",
    "    g = add(e, f)\n",
    "    return g\n",
    "fancy_func(1, 2, 3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "def add(a, b):\n",
      "    return a+b\n",
      "\n",
      "def fancy_func(a, b, c, d):\n",
      "    e = add(a, b)\n",
      "    f = add(c, d)\n",
      "    g = add(e, f)\n",
      "    return g\n",
      "\n",
      "print(fancy_func(1, 2, 3, 4))\n",
      "\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "def add_str():\n",
    "    return '''\n",
    "def add(a, b):\n",
    "    return a+b\n",
    "'''\n",
    "def fancy_func_str():\n",
    "    return '''\n",
    "def fancy_func(a, b, c, d):\n",
    "    e = add(a, b)\n",
    "    f = add(c, d)\n",
    "    g = add(e, f)\n",
    "    return g\n",
    "'''\n",
    "def evoke_str():\n",
    "    return add_str()+fancy_func_str()+'''\n",
    "print(fancy_func(1, 2, 3, 4))\n",
    "'''\n",
    "prog = evoke_str()\n",
    "print(prog)\n",
    "y = compile(prog, '', 'exec')\n",
    "exec(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.2 自动并行计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import torch\n",
    "assert torch.cuda.device_count() >= 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Benchmark():\n",
    "    def __init__(self, prefix=None):\n",
    "        # 打印的前置字符串\n",
    "        self.prefix = prefix + ' ' if prefix else ''\n",
    "    def __enter__(self):\n",
    "        self.start = time.time()\n",
    "    def __exit__(self, *args):\n",
    "        print('%stime: %.4f sec' % (self.prefix, time.time()-self.start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(x):\n",
    "    for _ in range(20000):\n",
    "        y = torch.mm(x, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_gpu1 = torch.rand(size=(100, 100), device='cuda:0')\n",
    "x_gpu2 = torch.rand(size=(100, 100), device='cuda:1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on GPU1. time: 0.8804 sec\n",
      "Run on GPU2. time: 0.9097 sec\n"
     ]
    }
   ],
   "source": [
    "with Benchmark('Run on GPU1.'):\n",
    "    run(x_gpu1)\n",
    "    torch.cuda.synchronize()\n",
    "with Benchmark('Run on GPU2.'):\n",
    "    run(x_gpu2)\n",
    "    torch.cuda.synchronize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on both GPU1 and GPU2 in parallel. time: 0.9803 sec\n"
     ]
    }
   ],
   "source": [
    "with Benchmark('Run on both GPU1 and GPU2 in parallel.'):\n",
    "    run(x_gpu1)\n",
    "    run(x_gpu2)\n",
    "    torch.cuda.synchronize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.3 多GPU计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Dec 24 15:33:15 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.57       Driver Version: 450.57       CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  TITAN V             Off  | 00000000:02:00.0 Off |                  N/A |\n",
      "| 32%   47C    P8    28W / 250W |   3750MiB / 12065MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  TITAN V             Off  | 00000000:82:00.0 Off |                  N/A |\n",
      "| 46%   63C    P2    44W / 250W |   4493MiB / 12066MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A     12463      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12472      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12478      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12485      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12491      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12497      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12526      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     12533      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     19274      C   /usr/bin/python                   305MiB |\n",
      "|    0   N/A  N/A     27538      C   ...3/envs/PyTorch/bin/python      999MiB |\n",
      "|    1   N/A  N/A      9866      C   python                            743MiB |\n",
      "|    1   N/A  N/A     12469      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12475      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12482      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12488      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12494      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12523      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12527      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     12537      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     19268      C   /usr/bin/python                   305MiB |\n",
      "|    1   N/A  N/A     27538      C   ...3/envs/PyTorch/bin/python      999MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=10, out_features=1, bias=True)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = torch.nn.Linear(10, 1).cuda()\n",
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataParallel(\n",
       "  (module): Linear(in_features=10, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = torch.nn.DataParallel(net)\n",
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(net.state_dict(), '7.3_model.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Error(s) in loading state_dict for Linear:\n\tMissing key(s) in state_dict: \"weight\", \"bias\". \n\tUnexpected key(s) in state_dict: \"module.weight\", \"module.bias\". ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-8254ddcc7a34>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mnew_net\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mnew_net\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'7.3_model.pt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/data2/wzy/SoftWare/anaconda3/envs/PyTorch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[0;34m(self, state_dict, strict)\u001b[0m\n\u001b[1;32m    775\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_msgs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    776\u001b[0m             raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n\u001b[0;32m--> 777\u001b[0;31m                                self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n\u001b[0m\u001b[1;32m    778\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_IncompatibleKeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmissing_keys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munexpected_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for Linear:\n\tMissing key(s) in state_dict: \"weight\", \"bias\". \n\tUnexpected key(s) in state_dict: \"module.weight\", \"module.bias\". "
     ]
    }
   ],
   "source": [
    "new_net = torch.nn.Linear(10, 1)\n",
    "new_net.load_state_dict(torch.load('7.3_model.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncompatibleKeys(missing_keys=[], unexpected_keys=[])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.save(net.module.state_dict(), '7.3_model.pt')\n",
    "new_net.load_state_dict(torch.load('7.3_model.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:PyTorch] *",
   "language": "python",
   "name": "conda-env-PyTorch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
